{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 17833 characters, 1453 unique.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\scipy\\sparse\\_index.py:126: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_arrayXarray(i, j, x)\n"
     ]
    }
   ],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "import numpy as np\n",
    "\n",
    "with open(r'D:\\课程\\博一下\\NLP\\越女剑.txt', 'r', encoding='utf-8') as f:\n",
    "    data = f.readlines()\n",
    "data=''.join(data)\n",
    "\n",
    "# Get unique chars\n",
    "chars = list(set(data))\n",
    "# Get doc length and charset size\n",
    "data_size, vocab_size = len(data), len(chars)\n",
    "print(f'data has {data_size} characters, {vocab_size} unique.')\n",
    "char_to_ix = { ch:i for i,ch in enumerate(chars)}\n",
    "ix_to_char = { i:ch for i,ch in enumerate(chars)}\n",
    "\n",
    "X_train = csr_matrix((len(data), len(chars)), dtype=np.int)\n",
    "char_id = np.array([chars.index(c) for c in data])\n",
    "X_train[np.arange(len(data)), char_id] = 1\n",
    "y_train = np.roll(char_id,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据集大小： (17833, 1453) (17833,)\n",
      "finetuning!\n",
      "当前学习率： 0.05\n",
      "epoch: 1 ----\n",
      "Running Avg Loss:0.09127378463745117 at iter: 1\n",
      "----\n",
      "本书来自www.com免费txt小说下载前后：“范贤弟，还要com\n",
      "　　清啸迅捷之极的远不随\n",
      "　　怔的瞧所握长剑在这人一张脸庞上了。\n",
      "　　范蠡心下\n",
      "　　霎时，任何其他的念头都不见夷光，跟着玩，他来到她，长剑中有两片的牡热的血同双手持大剑术，带着一\n",
      "　　袭价小\n",
      "　　春夜的凉亭中殷\n",
      "　　一千名甲士声三口。想剑士连声狂吼，不清澈，真是愤怒。范蠡叫道：“我正是谁教什么？”断了顺水漂流，连许百片的声朵小红花\n",
      "epoch: 1 ----\n",
      "Running Avg Loss:2.8294172782599927 at iter: 501\n",
      "----\n",
      "是便是不听，心。”阿青三名年青应道：“我子……我要杀的戳向和羊少女早知道：“我这位说道：“乡下姑娘，我赔你你为甚么杀我要一你家里有大草地，我家里有大草地，我家里有大草地，我家里有大草地，我家里有大草地，我家里有大草地，我家里有大草地，我家里有大草地礼。”\n",
      "　　老白，乖老白吃了来吃\n",
      "　　阿青扑，这大好吃，我\n",
      "　　天真中飘去说道：“你来杀，我赔你不赔你真是个气，双明的瞧挥救国，你不许他们把老白的“我家\n",
      "当前学习率： 0.025\n",
      "epoch: 2 ----\n",
      "Running Avg Loss:2.8287259490489958 at iter: 1001\n",
      "----\n",
      "说道：“喝道：“着他脑袋砍了出来。\n",
      "　　赠你师兄的，只五口宝光，不止宫六七剑了越国。第九术却又有好，连输乃敝国新铸，好人家中，第一之极是的剑法，事鬼神，决不便派自三\n",
      "　　币，师曾能万剑斩口的。”勾践道：“原来将一惊又有身子，当二人。”\n",
      "　　勾践道：“这等铸剑则大夫之是咩咩的剑。\n",
      "　　薛烛。\n",
      "　　勾践道：“劲越王但根竹棒便前阿青衣剑。”薛烛道：“口夫无实了其是楚国吴国，后剑。”勾践道：“坐\n",
      "　　我要\n",
      "当前学习率： 0.016666666666666666\n",
      "epoch: 3 ----\n",
      "Running Avg Loss:2.149970999956131 at iter: 1501\n",
      "----\n",
      "笑道：“衣剑士嘿嘿一起中，躬身行过无不来找他。青衣剑士腕。他长剑，将人脉管中一子缝了神。不\n",
      "　　这声音乐的锦衫剑法过去，显然人必须尽快礼所住的神，心口感到这样。\n",
      "　　青衣剑入了他的青衣剑一挥，将剑乃无不费中得他忍来，\n",
      "　　一连剑劲青\n",
      "　　剑向地。青衣剑士柄三步，只听得她喝道：“你也回里，而立。锦衫剑士，一道好人都齐，\n",
      "　　“范蠡，两人身子缝了条手举，剑不住倒也不由得剑锋锐长，从前这剑一吴士提长剑道\n",
      "epoch: 3 ----\n",
      "Running Avg Loss:1.6079846745729447 at iter: 2001\n",
      "----\n",
      "能跟他起吴越剑士又啊，我十道：“\n",
      "　　想是西施，向那开了声道：“那明亮，对方的大范蠡自己手他的的吴国剑士的\n",
      "　　丧，是我从左上得他见这少女不会忘着她白雪久没吴士的竹棒已声\n",
      "　　，一根竹棒刺去。\n",
      "　　范蠡我\n",
      "　　范蠡我妈和我跟她的西施，不要阿他奇听了起吃去之阿青拆叫，倒在清澈的，所用兵刃.cr173.com\n",
      "　　啊名青衣剑士从下。众位剑士道：“我见这阿青赶羊玩了道：“好老白的白公公\n",
      "　　她的长种，连\n",
      "当前学习率： 0.0125\n",
      "epoch: 4 ----\n",
      "Running Avg Loss:1.4300035352110863 at iter: 2501\n",
      "----\n",
      "要溶在会稽，身子尚子胥奇怪，有十，也是一口么，都是在危叫，风师兄承受。”\n",
      "　　勾践召薛烛道：“伍子胥虽然诡计多端，别说话，也不住”范蠡道：“小人世大夫：为楚国所铸，有如狼嗥，阖庐以已齐百姓采锡和西施。\n",
      "　　他笑更多少次剑匠，他笑了自须收：“小\n",
      "　　师兄的纯钧、湛卢、胜邪以如此了，\n",
      "　　替风师哥来召小人曾言道师，听得伍子胥胜为楚，是言谈还哈哈大夫共。风师兄言道吴越将花园中透过\n",
      "　　杀看。”勾践道：“\n",
      "当前学习率： 0.010000000000000002\n",
      "epoch: 5 ----\n",
      "Running Avg Loss:1.194021679610014 at iter: 3001\n",
      "----\n",
      "，将他右臂齐肩削落，背\n",
      "　两名青衣剑士的剑法，飞，勾践\n",
      "　　定了，却也有十名越国剑士长剑剑出。突然去。那在地下。那名卫士在可边。锦衫剑士道：“我\n",
      "　　住了人剑，\n",
      "　　若不见，又得两名青衣剑士虽是刺中，另外两名青衣剑士齐声对过剑\n",
      "　　半干杀的声。\n",
      "　　“好人。”薛烛磕头道：“若是手守势的两场，不住敌手削断。锦衫剑士后，另外两名青衣剑劈为专下的对两个越王昔日为夫差派\n",
      "　　越宫门外，月剑不肯来。我青衣剑\n",
      "epoch: 5 ----\n",
      "Running Avg Loss:0.9872925525903702 at iter: 3501\n",
      "----\n",
      "咯咯娇笑，\n",
      "　　我们已失了文种，显然\n",
      "　　之下敌大极厚礼，在长廊彼端，音乐般的脚下击，吴国剑士\n",
      "　　玩，女竹棒蹲乃是啊。这人来跟我玩了。\n",
      "　　这见过了良久。当真是剑之在山。\n",
      "　　范蠡手下的第九天下吴越剑士，这次\n",
      "　　让同伴以高手。但常兵法战之战，请医生医治伤目道：“端，……她天真烂漫，既是叫道：“我叫阿青与站\n",
      "身来。他铸看出破窗\n",
      "　　从头鲜艳的好棒笑。他手矫笑了？”\n",
      "　　阿青的锦衫剑士和道：“范大\n",
      "训练完成\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEICAYAAAB25L6yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2dd3wUdfrHP08qJAFCCUgPTarUgHREEGknnnqKnvX0sJ2g3ukPxYJ6nqgnKuqpnFgPwS4qXUCK0gKEJj2EFgIJEAgkpH5/f+zOZnZ3Znd2s7Pz3cnzfr14sWXKs5OZzzzzfJ/v85AQAgzDMIy8RFltAMMwDOMbFmqGYRjJYaFmGIaRHBZqhmEYyWGhZhiGkRwWaoZhGMlhoWYYhpEcFmomIiGiLCIarvF5MhG9S0Q5RFRIRNuJ6C6PZQYS0W9EdJaIThPRr0TU2/ldHBG9RkRHieg8ER0kotfD9bsYRosYqw1gmFBBRHEAfgZwEkA/AEcBDAPwCRHVFUJMJ6LaAH4CcD+ALwHEARgEoNi5mScApAHoA+A4gJYABofzdzCMJyzUjJ24DUALAEOEEBecny0iookAZhHRBwAuBQAhxBzn90UAlqi20RvAd0KIbOf7LOc/hrEMDn0wduIqAAtVIq3wDYAacHjZewGUE9EnRDSKiOp6LLsOwKNE9AARXUZEZL7ZDOMbFmrGTjSAI1zhhhCiDEAegAZCiHMABgIQAP4LIJeIfiCiRs7FXwLwMoA/A0gHcIyI7giH8QyjBws1YyfyADT2/JCIYuAQ8TwAEELsEkLcKYRoBqALgCYA3nB+Vy6EeEcIMQBAMoAXAXxIRB3D9BsYxgsWasZO/AxgFBElenx+PRyDhes8VxBC7AbwMRyC7fldkRDiHQBnAHQKubUMYxAWaiaSiSWiGso/AJ/BkenxFRGlElEsEV0NYAaAqUKIs0TUgYj+TkTNAICImgO4GU4RJ6KHiegKIqpJRDHOsEctAFss+YUMA876YCKbBR7vXwQwHI4483oAtQFkApgihPjAuUwBgMvhGDBMBpAPR7reY87viwC8BqAtHHHsvQCuF0Jkmvg7GMYnxI0DGIZh5IZDHwzDMJLDQs0wDCM5LNQMwzCSw0LNMAwjOaZkfTRo0ECkpqaasWmGYRhbsmnTpjwhRIrWd4aEmogeAXAPHOlK2wHcJYS4qLd8amoq0tPTg7GVYRimWkJEh/S+8xv6IKKmACYCSBNCdAEQDWB86MxjGIZhfGE0Rh0DoKazZkICgGw/yzMMwzAhwq9QCyGOAfg3gMNwVCY7K4RY4rkcEU0gonQiSs/NzQ29pQzDMNUUI6GPugDGAWgFR5WxRCK61XM5IcRMIUSaECItJUUzHs4wDMMEgZHQx3AAB4UQuUKIUgDfAuhvrlkMwzCMghGhPgygLxElOLtdDAOwy1yzGIZhGAUjMer1AL4GsBmO1LwoADNNtothGIZxYijrQwjxrBCigxCiixDiNiFEsf+1GIaxC/mFJfhpGyd7WQXXo2YYxi8PzdmC1fvy0K1ZMprXS7DanGoH1/pgGMYvx84UAQBKyisstqR6wkLNMAwjOSzUDMMwksNCzTAMIzks1AzDMJLDQs0wDCM5LNQMwzCSw0LNMAwjOSzUDMMwksNCzTAMIzks1AzDMJLDQs0wDCM5LNQMwzCSw0LNMIxfhNUGVHNYqBmGYSSHhZphGL+Q1QZUc4x0IW9PRBmqf+eI6OFwGMcwDMMY6PAihNgDoDsAEFE0gGMAvjPZLoZhGMZJoKGPYQAOCCEOmWEMwzAM402gQj0ewBytL4hoAhGlE1F6bm5u1S1jGIZhAAQg1EQUB+AaAF9pfS+EmCmESBNCpKWkpITKPoZhJIDT86wlEI96FIDNQogTZhnDMAzDeBOIUN8MnbAHwzD2htPzrMWQUBNRAoCrAHxrrjkMwzCMJ37T8wBACFEIoL7JtjAMwzAa8MxEhmEYyWGhZhiGkRwWaoZhGMlhoWYYxi/VMY96x7GzmLnqgNVmADA4mMgwDFPdGPvWGgDAhMFtLLaEPWqGYQzAedTWwkLNMAwjOSzUDMMwksNCzTAMIzks1AzDMJLDQs0wjF+qY3qeTLBQMwxjGM7+sAYWaoZhDMOetTWwUDMM4xf2pK2FhZphGEZyWKgZhmEkx2iHl2Qi+pqIdhPRLiLqZ7ZhDMMwjAOjRZneBLBICHGDsxt5gok2MQwjGTyIaC1+PWoiqg1gMIBZACCEKBFC5JttWFXJyruAr9KPWG0Gw9gKHlS0BiOhj9YAcgF8RERbiOgDIkr0XIiIJhBROhGl5+bmhtzQQPnDW2vw2NfbrDaDYWwFe9bWYESoYwD0BPCuEKIHgAsAJnsuJISYKYRIE0KkpaSkhNjMwCkoLrPaBIaxDexJW4sRoT4K4KgQYr3z/ddwCDfDMAwTBvwKtRAiB8ARImrv/GgYgN9NtYphGIZxYTTr4yEAs50ZH5kA7jLPJIZhGEaNIaEWQmQASDPZFoZhJIUHEa3F9jMTheBTjGFCBQ8qWoPthZphGCbSsb1Qs0PNMKGDLydrsL9QW20Aw9gADnlYi+2FmmEYJtKxvVDzYCLDMJGO7YWaYRgm0rG9ULM/zTBVh68ja7G/UPMZxjAhgwcVrcH2Qs0wTOhgv8cabC/UwqRT62xhKUrLK0zZNsPIBnvS1mJ7oTaLbs8vwUOfb7HaDIZhqgG2F2ozY9SLduaYt3GGYRgnthdqhmGYSIeFmmEYv/AgorXYXqg5PY9hQgcPKlqD/YWafQGGYSIcQ0JNRFlEtJ2IMogo3WyjmOD4Kv0Inp23w2ozGBvDbo81GO2ZCABDhRB5plliEtUp9PHY19sAAM+N62KxJYzd4JCHtdg+9MEwDBPpGBVqAWAJEW0ioglaCxDRBCJKJ6L03Nzc0FlYRWRxqD9bm4WvNx212gyGYSIQo6GPAUKIbCJqCGApEe0WQqxSLyCEmAlgJgCkpaXJoo/S1KN+et5OAMANvZq5fV5eIRAdxQ+WjNzIcRVVXwx51EKIbOf/JwF8B6CPmUaFkuW7TyI967TVZmiyYPtxtHlyAfafPG+1KQxjCHYprMGvUBNRIhHVUl4DGAEgYlILJs3NwA3vrbXaDE0WbD8OANiZfdZiSxiGkRkjoY9GAL4jImX5z4UQi0y1KgQQyZ/xoZjnPLYMIz2SX1K2xa9QCyEyAXQLgy3VD+dZzzLNyA6fo9ZSrdLz7vtsE6Yt3G21GV6wQ80wjC9sK9Ra2rdoZw7eW3kAOWcvYvI321BSFlzh/1BlkvD0doZhjGBbofbFlO+2Y+7GI1i5N7T53ntyCjDlu+2oqDAmwIreR7FLzUgOuxTWUi2FWiFYedRyqC8Ul+HqN1Zh9vrDOHqmKKDtsEwzkQKfq9ZgW6EOdyZF52cXB7wOhz4YhjGCbYXaF2bLoxEBLiwpw6ZD+QB4MJFhGN8EUj0vosk7XxyybVVF6L/YeBj/98129G9TX2UTKzUTGfAzoDVUG6H+YuMR12uzZVEdw/5xazbiYyofXP637jAA4LcDpyrtYZ1mJIdPUWuxlVDvzjmHez/bhHkPDvA6sV5dvMf12vzQRyUPzdnid3m+CBiG8YWtYtRvLd+PQ6cKsXqfsf4GwXqy/vKofX2vtU+eQs4w8iJDBU5bCXWgqI9/YUkZSsuDmwDDMHbHeqmq3thLqFVnUyBOqhACnZ5ZjDs+3BDoboL63hP2p5lIgc9Va7CXUDsxKtLKcp+uPQTAfYCvKigzE79KP+JnSXc7GIZhtIgooT5x7iJW7Dmp+32wE0h+3V8Z0xZCYO+JAq9lbvnvOqROno8jpwtx5HShz+29sngPLpaWu5rN+oOFmokUOARiDREl1H9851fc9dFGv8tRgA9oaqH8PuMYRry+Cst2nXBbRvG2B72yAle+ttJrG/UT41yvV+zWv5kwoWVPTgF+zz5ntRm2h30Ja4kooc4+e9HQciv2nERpeXD3/l3HHd70vgDbY9WMiza03Laj3t1cOOsjeK5+YxVGz1httRmMjZEg6cO4UBNRNBFtIaKfzDQoFATa7TtQD1xN6uT5KC4rd/PKBYCKAP66LNMMw/giEI96EoBdZhkSTpZ7hCaq6tCeOl/iVqpUCAGDlU4to7xCoMPTC/HFxsNWm8JEAJKfzrbHkFATUTMAYwB8YK45VSPYRxS1UCsvpy3cjaNnfA8aum1DbQcCS5K3IvRRVFqOi6UVeP7H38O+byZyieSnv8OnCpFxJN9qM4LCqEf9BoDHAejOCCGiCUSUTkTpubmhLcjvSahnCqlDH7tyKjM+vtxoLL1OwF1shUBAHrUMM58Yxu4MfnUFrn3nV6vNCAq/Qk1EYwGcFEJs8rWcEGKmECJNCJGWkpISMgMDIRi9m7FsH+ZvP+56vyrIri+ensaHaw4aXvet5fuD2ifDMNUDIx71AADXEFEWgLkAriSi/5lqlR9C5YASAdOX7tXfj8Ht5BUUeyn1m8v2GbZjZ7Z3JggTeSzemYPcgtCV05WR6vjsJ8Nv9ivUQognhBDNhBCpAMYDWC6EuNV0y3zZFKLtZBwJjUCOe+dXZOZeCHp97pkY+RSVlOPezzbhtlnrrTbFFPgMtZaIyqNWCFVMd4Yfr/ezdYdCuj89+CKIfMqd58hhP7NWGSYYAhJqIcQvQoixZhljlO+2HAvLfvILS3GxtNz0/VwoKcfWCB2NZqoHMjz+V2ci0qN+95cDmp+b1Sw2HEkZ4yJ0NJqpXvDTnzVEpFCHEw4fM0bg04Qxk4gUaj0H1yzPt5zznBmGsZCI7JmoN7i35PcTmp9XhUEvr0BcTETezxgL4Hu6/XDojbXPTBEp1OHkpM3zYpnQUF1CZHwfsoaIdBX5ZGGY8FJN7kPSEplCzUrtYuuRfDw4ezPKAyzXx/VFzMGszCOrseevihwiUqgB4GxhKVInz8f3YcqplpUHZm/G/O3HkZ1fFNT6fAGGhqrUNI8kqsevlI+IFeqsU44p27MCKH7EVMICHVrs6kkzchCRQi0gqs3gjT+U4xBoJENZng8jw/hGhltwZAq16sjZyZM5fjbw8IXZN6yLpeW49YP1mp3Zg6G0vAL5hSUh2ZZMcMifMZOIFWolJminC+Rvn28Jet2Ab1gGF9+YdRpr9ufhuR93Bm6UBhPnbEH355eGZFsyYqfzUQub/zxpiUihBoJ/5JeZM0F4mkqJVLOOQ6gHyRbuyAEAZOaex4XispBumzEPDpFZS8QKtScbs05bbUKVOX+xDAUXSwNaR7mA1Drd9skFeHbeDp/rBeqBG7kRlJVXoLxCoLCkDJdOWYjFO3N0l73ytZW46+ONAdkQag7mXcB7K7ULfBnlYmk5ysorbO9p2v33yU5ECHVRSTlSJ893+0ydN/zt5qP403trw21WyDlZUIzLpi5B6uT5OHU+sBmR6rzosgqBT9YeColN6ieXMxdKUOEjX7vtlIUY+cYqHD5diJLyCry2ZI/PbW84aO3NdfzMtZi2cDfOBXhzVNPh6UX466fprvd2FzT2rK1BaqEuLa/A/9YdwmmPkIAQAi/85OieXVBcastazvtOnve7jBACpy44jk2gAmE0VKJcmHnni9HjhaV4/Wf91mWAw+5IySkuLA5NrfEVe8xt5sxYiwzhVamF+oPVB/HU9zvwhUc3cAEg/dAZAI5wAdkwV8/IL3pvZSYKLjrivKadTE5D8pwe/pKdlYWvdh0/hzX78kzacfgIxbHjmZ6MmRjpQl6DiDYQ0VYi2klEz4XDMACuNK5zRfqPpgLVpyCOJyt2n1S98y8UP/9+AqedHvjKILutq2Pbo95cjVs1egQWhaErTkiI0PNmT05B0DNRmcjESPW8YgBXCiHOE1EsgDVEtFAIsc5k21yS4Nn8VR2fzi8stWVzWK2nhIul5cg4ko++resDcBfNkjLfQl1wsRT3fJqObs2T8cldvfHwFxl+bRg+fSUKnZkZytYPnSpEztmLuKRODd31rq2G3WrC6U9f/cYqAEDWtDFh3CtjJUa6kAshhBIwjXX+C+tzXn6Re4zas/RodfEuHv96G8bPXId9GpNP/MWOy8odf7JDpy6guKzC0P72nzyP7LMX3T4rLqtA35eWGVpfiQZcLC3HmQv2m+Siic0jIDb/edJiqB41EUUD2ASgLYB3hBBez7tENAHABABo0aJFSIxT4n7fbvZdeEnJzbUjB/MuIDaasDHrNH7Ymg0AOFPoHQoy+2aV77HPkU6vTuFkgbugA46BxT+8tQYXSsqQmXsB26eOMNXGoAlCfS4UlyExvvqUc7ffM2tkYWgwUQhRLoToDqAZgD5E1EVjmZlCiDQhRFpKSkpIjKvO4zNK5GPov3/BwJdXYKUqs0C5gamPj1bW3FkfsX2FCyXlWLD9eMD27c6p9OpPnruIPi9qe9nbj51FZq6jgNYHq+UqoBWs+PywNRudn12M37PPuT7TOlfzC0swL8Me1R2r8aUoRZmKgLI+hBD5AH4BMNIUa2xIi3oJQa3nKSLfZ2S7XiuirD59tPKbe71gbKr2A7M3IzPXfzqgHka74Mg6lBDohfjLHscg7s7ss17flZRXoNUT8zEv4xgemrMFk+Zm4PCpwpDYKQOS/gltj5GsjxQiSna+rglgOIDdZhsG2OMuPrxjo6DWe39Vpu53+3PPo6zcPc6s1YC3rEKgqMQxc+6jXx3ebKlOfLqwxJGpsfbAqYAn2xgVYDPzq1Mnz8fHvwbmsWsN2K7LPIWpP/iua+LvdwgBvLp4D4474/vFZRGSBcNIi5EgW2MAnzjj1FEAvhRC/GSuWcBv+/Pw+frDZu/GdJITYgEAjwy/1O+An5qlv5/QjTs//f0OHMq74Ja7qzdjsOMzizCsQ0Msc6byXSjRFo2xb61xvW7XMAlLHx1i2FajmO1Rv7Z0L+4c0KpK2xg/05HMNPWazn6XFbpvHL9Va3o/wwSDX6EWQmwD0CMMtrhxywfe+bmRSEJcNPa9OAoxURSQUAPAo1/qp9BtOnzG7f1FH7nLy9zyrf3H/o3MilQzZsYa/wshdI/N5RUCpeUVqBEbbcr2/aF1w3l/VdVqhjCML6SemWgXYqOj3B6zjYZD1mXq18KIJnITXM80Oj2MerVHToc+rhoVFRopnTh3Czo8vcjr82BnqPq7cd3w7m9etWY8+c8v7kKtDo/YaVBc/VOOny3iGZlhgoXaBJr4mAwSKsye5DPolRUh3+YBDU/9fHGZz6cBLeZv085SCfSQGF1eKVfghR+NknXwNBg8f8runHPo99JyfPxblhXmVDukE+plu07gnk+sLX9ZVYz4GIseHoRv7u8f9D42ZJ1GRhDFqITQznkOB99qNCLu8uxijHpztc/1Dp8qxM+/n/C5DBD8zStQn9DIXtSmHMw7j/WZpzBj2b6guvjIgOcxyspzpFyuPXAqqO0VXCzFJr0boGTI8NAgXcb+3Z+k+18oQP4yoBU+DDAjoCqo/7CjLmusuUyHS2rjrMbElXCw9ah3WpmVHHRe9AoPzN6Ei6UV+PDO3gAcU9lLyiv8Tpk2KtNni0qRHoL65f7S+pTwx33/2+z6bOXe3CrdoK0mVA8J9/9vM9bsz8PO566uVhOHgqVaHKGb+zQPr1CrLuCmyTX1F7To0dhXTWkZWLDdfaZpSbmxKe9GHeq/f7kVP++q9NADjbMa2Q/pLFeok3UTeVTt5N121PE0qJQ2YHwjXejDDNo1quV3mY+c3lsoMKqDVsUwyyUX6uAxdkA9ww/BHg1f+m7H0rvu2PUc8iY9y/oQjXRCbYfzW+8CvjGtGQCgTUpiGK3xxq5CbfTcifHIPgk0BlmViTtnQ9iBfeH24wEPxDKBI8Mxlk+obbDj/m3qa37esr5DoBVvy6rfqjWL0WqKQhASCPZ4BlvLYfsx/Vi/I/ThbZHRNEoj3D97c8i6wweK0QqMdkCGq0U6oY702tKLHh6EN8d397mM8hOtejyW0aPu+MyiKtulnDtl5RWYvnSvbqPgUP362T5mzuYbKIgVCo7lhzeDRzl2k+b6r2duZDuRgAy54tIJdVkVL9bY6ODEL5C1WvsIXXS4pLb08UlZO7ZXVPGCUA77D1uzMWPZPvx7sXZzXa/daOx2lY8OOEb+vKcvlITlicnqM03yUz0keJ4eM1cdwO6cc5rLmoV0Ql1V3r6lp9UmAABa1k/A7f1aan5HHv+Hm18kbcZ65kKJm/dywdldRsHTs/F8rxxP5bH8Yqmxx3Ot28PO7HMoLiv3Sh0MBKMitvVIPgZMW16lbuj+KCuvwI9bs6vsHVYDXfbLvxbsNlw2IVTYTqhretR/MAP1ydq9ebLmMisfG4rnx7mX7fYSFj7r3bju3d9czREA4J0V+92+99QYz4cv5Ukm43C+8732fjxj0lradbG0HO2fWoSh//7F6zujf7ed2dpel2flw+lL9+JYfhE2BZFdYNSWmasz8dCcLW7HlwmecIcPpRLqUP74zk1qB7R8MOGKbs2T8dV9/Vzv7xlorHKbK0bN/okbR88UuU0zL/UQNM/QiNaNb++JAnyRfsT5Xvv47jjmLqBag4k5Pgb95mw4ovudETynXVflhm10VeX3HD0TmTMjraSiQmD2+kNe52M4kUqoPwrRpJSfHx2CuRP6hmRbWigC8OoNXREbXXkIx3ZrYmh9CcYmpEV9r/7v6oPYqpomf9xDPDccdI+1E7m3DNMSwMe/3ur1mdbfQxF7M/DsvKPs/8NfDwbcZCDQwfdXF+/Bt5uPBrROILyyaLftvPYv0o9gync7LO1QJJVQHw5BxTYBoG3DJNSqERvQegTt/OaEOO9QiqvOsMcF3qpBYPnR6mvs/dt6BbSuXfH0mudurBTM22a5l771LIVLILfBZC0J+zLdPJGqKqv35WHwq4EVwzLetKGSlT4GSqvKf345gIlztpi2fStQbv75IcyBDxSphPrTtYfCtq+b+7g34I2PiULdhDiv5bSuA72Lo05NYzcHrfXbNkwytK7d+WKjuyd7QNUiLNdPyy8iuD3heBb90RtIM/qAo9V6KxhCG/AKb/jM17GavsQ9y+bkuYtI++dS7DtRoLNGZKBcr1Y+CBtpxdWciFYQ0S4i2klEk8JhWLDoXYwjO1/iuaTbuz6t6mFA2wZe6/l6tAx0ooSvkAeHQxycuuDutajDGxdKyrF6n4+0Obh3qtmdU4DbZq13VQvUK1daUlaB1Mnz8fIi3x3mxsxYg8KSMp/LBENZRfCxT18e9Z6cAlz3n19Nsdm1f9WNYsZy98Hfxb+fQN75EnwU4aVQK5+ghWU51UY86jIAfxdCdATQF8CDRNTJXLNCz/SbuuHO/qmu957XBhFh0rB2XiEIrQuhqoOAyvrqbVs9rTxSuG3WBt3vsjTiu6v35WHWat/9Iv/u7KTz7i/+u7RcKA79dOJdx83xOP+1YBc2H87H+szThl2K1Mnz/faMVKPnrAx6ZTlWmxhisQK9LJ5w4FeohRDHhRCbna8LAOwC0NRsw4JFfdrERhPucOYyJ8TFuPXB0zrBoqIITerU9PpMd19VvLmqBd9o1kk7DpEEzPurMrFw+3FE6/wtNx82XtfbaCU/f6zPPGWoGW/q5PmY/M023e99nTW66Yl+zltfzQCEAJ74drvvDQA4croISwzUEI8ElGvztwOncOJcYI2fQ0VAMWoiSoWjf6JXQ0MimkBE6USUnpsbvjtpvUTvuLLCvhdH4zmPXGYFvUxATwFXzvUXrq3cjucFsOSRwVj08CC/tobioek9HnQMio9+zUJMkLNW1fhK2zMMEW6auQ5Tf/zd8dbP4nM36megGLm/F5aUY8Weyr6ZmXmB9cRcsjPHNfHnbFEJ5myonDpPIAghfGaShKpJtRACG7NOhz38oD7E98/eFNZ9KxiuR01ESQC+AfCwEMLrGUAIMRPATABIS0sL25F879Ze6NOqHu74cIOh0ezb+7VEg6R4HNJJg/I8B5rVTcCZwrNomuzdXktZ9lIDZVTVVNb6CGg1AEBctFTjvxFDhRD4IaPqaWPXv/tbCKwJHb7CcMo3k7/ZhgLVLM8dx84h5+xFXKJqGbdmXx5unaXdUHrCZ/riJCDwy95cPPqld9pjKBn0ynLUrhGLndnn8PpN3fDHHs1M3Z8eWwJ4+golhq56IoqFQ6RnCyG+NdekwOjTqp77B35uEc+P64KJw9rp3pU7Nq6Nfq0rq981SNL32KsKT3cJH+VC4JMwZhWFC/XNvris3G3Wo/LIXlDsPZjY96Vlbg2MZ63JdPt+d8457D9pLHZ+rgoFqIwOyB85XeSKEes5WWr2nzyPTyJ8EFONkawPAjALwC4hxHTzTQqOQL1TvQJAcTFR+PTuPl6fay0ebHlMPYzEnzk7JDhkKhjoeap6ZrpoMS/jGAZMW+41IUZ93rd/ahHGz1xn2I4T5yrDOJ5jJCPfWI3h01f53cbinSdM6VjvCyOD+de8vQbPBjAo6ovNh/Wn9ldUhCcTxIhHPQDAbQCuJKIM57/RJttlOr4uXH+ngXJSV3kw0ePiWProEL99AataYa66ct7EgkeBkp1fOY1baUnlj0lzM3Asv8hrQoznOaTbMd0PVXm6W5sZXINbwDzHI5Qtz3xpResnF+CB2Zv1FwgRfmPUQog1iKCndKNe7qB2DXSnuvrLwPjPn3ti1ppMdGwcWD0Rz5MymINqllCP6doY87cdN2XbMnAgN/gqeKEmR+XJ6nXiPnK6EAUX/ec/+8z68LOucibd/79NWLb7pM9lFbSyHoxOY992NB9FJeXILyo19Nv0CEcxs0Cm2S/ckeN/oSpim+a2j13dHtn5ReidWs//wgBu6NUMV3VqhO7PL/X6Tn0eaHnPrRok4p/XXha0rVU5z8x6hJ8wqHVIhXr140Mx6JXApkNXF1bvy3O9fs6Z+eFJVY7diNdX4seHBhoStJ3ZZwMSGi3v0ahQX/P2r16fVfV0Lq8QiCJ950oIEVTBtfWZctVsj+gUgpevrxTLzk3qYMkjQwzX+CAiJGtMGXd8p3pdJQvd8Ur9C2LjZsXD9HKMg6V5vQR0aar/xNGrZd2Q7s9O6HWm0UJLhPaeOI+vDNQ0IQCr9ub5Xc4f4a4qp/7FbURcA0sAABZJSURBVJ5cgGfm6ceigw2BhHr8qapIJdTX99RPuXnums6YeGVb1/vXb+qGG9Oam2KH6R1alJ6JQexHz6P+WlVuVRa+e2CA7ndaxa4YB5kBhGkIDmGfNNe9ENJT3++AETdj+W7fk1LU8XQ9fjvgP0a9V6feRzCOh6v2hnPdz9bpZ/O8uGBXwNs3Smauez76mn152HXcnNmLUgm1rxTh+klxeHREe9f7P/ZoZrqgPjm6A2653FG8qUvTOlXeXlK8I9KUWAWR0otRpxkM+eihPpS14kMTEYv18QeVvV2ZlYx7xztEoAeRo5jZPI0c8WN+RPaHrdnY6KdZQf9pyw3b4oujZ3TmLfhYp7S8AlN/2IlT57VnA6qdlgHTlmOdxqBmnp9CXgCQd74Y32w6itTJ83HGmYFj5P5x5Wsr3d7fOms9Rr252v+KQSBVjDo+Rl/AFu3IwdiuTfDD3wbg1Hnzyw2qsy/8ZWIY5fZ+qSivELhrgLEGA7VrxOCcx6CLVgrfpGHtqmybOs6oN22+dUpiQN6eJ1ufHYEfMo7haR+PqkxgEIAsnXZh/ooxhbNapV5KXdo/f8YV7VPw8V3eKbGLduTg49+ydMuLqhuNHMsvwsuLdns9xZ0pLEFhSRkS4rSlbsnOHLcJPYdOF6JuYlzAgY8Sk7uyS+VR14jVN2dcd0d5ka7NkjG0Q8NwmeTGzNt6Yf7EgUGvHxcThXuHtEFcjLHDrnVXj9HwUidqCLXnRKBGteMN70s94UfN3L/2RZM63jM0L21krP5InZqxSHR66zJ0drYD32dk46tN2vFoIxNDZEDdw3P70bNInTwfB/MuuJ4ey3Varnk+XWrdCjZmncFYH/0NNx3WLoUb6On54OfmpuhJJdTRUfrmmDlD0CgjOl+Czk2qHgIxinIibp86Ag8ObYOGtbzFdu6Evq6BwKl/6OSqax3vcTNYNGmwz32pB0/eGN9dM4bcsHYN/PbEMHxzf3+3zztcUhv3DmkNALiuR1PM1KhH8uToDgDU8UWf5jA2Y0OWsSyKP7ztENWvN1XWN9G7qZdVeA7Oa3vtmXkXUKEzuBPtsU6wp+VSkwtQSSXUiljUTfDO3Ah1VoLCNd2aGPZww01Np1hGEeGxqztgw5ThACpFDwD6qrzfOwe0whcT+mLC4NYY2t79qaOuTvGqsV0bY/PTV7kJZ43YaNwzqLXbcs3qVlYV9MzYeGFcFyTEOjzlpnVrYoRX7W+gdQPHDUR5BJZtVJ0xFyMlZNV8vyUbRc6MDb0zZdkud3FUFOK0xkxPvcJWnrqiXAeynZ9SKdT7Kx31BhZMqqxEpzSpDbQ3nFFm3NwDe/85ypRtV5Uv7u2HyaM6uMIFChMGt9Fdp35SPJ4c3dF1Aqa1rIuFzuPpGbZ4YlQHvH1LT7cKhK4QiYcXM+9B/QyOOho3Vk88/3zsUTNqNh8+4+b1HssvwmRnOdWzhe7pisq5NGluhubnPV/wnhtxINe7YuD7Kw/gLY9mBy4kOz+lGkxUaKyqCa387eyaJPDoVZfiivYpmt+1SUlCmyHa8d9eLetiRKdGuttVHhc7NamtOYMy/anhaJDkHUqpn+j4zPNJMT7Wd6aKv7+P5/d6gztM9eS6/+hXJVyz3z3X+5VFe/DKoj1ey/mqATJrjXft73dXenv5j3yRgVWPD/VlqiVI5VFrodxlzfKorWbisHbo2izZ7bN3bumJQe2824Kp+eb+/rh3iL5nraA1yxKAl0grHq4yTOA5UBPqwb/E+OqXR+3rxspUna0G66b84qzN7RmfBiobbEvmUMsl1C3qJaBrM8dg3Z39U3Fdj6b4U5pjEkxjjWwDuzKma2N8dvflVdpGjxaOOPKgdtreuidKTK4yhuzg2u5NMHFYO1cOeFVRx/5eub5rSLYpA9OuC76kABMaissqsMVHpTuFOz/aCMB31ULZspKkEuo6NWNdnt7Uazpj+k3dcffAVsj812jd6d6MNt2aJ2PX8yMxXOXFvX9bLyTFx+C9W3t6LR/jdKVr1XAIsvJ36N+mAR696lK/E1Qud6YD9tVJ7fN8LCUAN/Y2Z2apFYz36GqvhVyXvj05csb/TEoA2HTIdxaKbH8rqQKFWiOtRGTb+LTZ1PRIsevStA52PHe15rIdG9fClNEdcW0PR776nf1T0SApDn/o2sTQvi5vXR+7XxiJGn5i2WbSqXFt/G7SFN5QIJmTZksmztnifyHAayKZ7EjlUQsRQfVUbQYR4a+DWyPFmasdHUUY172pz+a+nmiJ9Jr/G4o/9miKAW31Y+6e6ZGeedqePDWmIwBH82I1oeiJaC6s1LLga8xr/8kC6W6qUgk1YN/sjupKs7oJeP2m7i4xFq4sHscf+pO/9MGyR4e4lt8+dYRbnrZW/vw9g1oja9oYLH7YfRJPjIGbymca3XvChWwXf3XG15ny24FT0t1SpRJqPpHtj0uone+HXJqC5vUSsOqxofjugf6uMrX7XhyFzH9VNhL6dfKVXtvyFPEUjZmbnpRVCGx++ircf0Ub3NArNA1SV/zjCgDA27f0cPv8879ejg1Thrnec3ceefjCR2f3Y/lFAZWaDQdGeiZ+SEQniWiH2cYoeQeM/Bz4V2i7sbWon+DKVAEclffUYZf6iXH4/K+XI+OZq1yfqXO9p4zu6BoQBRwpjlpUVAjUS4zD/43sgPqqsgRaA6yGba+XAAAY27UJDr5UeVzaNayFhrVquCowetasYKxj/nb9Jhnvr8x0qz8iA0Y86o8BjDTZDhcc+ogMgp3SH6xWRRGhf5sGbtk/SgpVUnwM7hnUyk3Yx3RtrLkddX2IR4Zf6no9sktjr4lH26aOMGhb5Wt1doxnXZM0bpbABIlfoRZCrAIQlr40suUuMiYSoM5r3cCb10tAn9R6+PDO3iAitxooeqinKXsOfnqWqqwdQLcgzc9drxz7rJ8UhwFttdMXGcYXIYtRE9EEIkonovTc3OAfG9ihZtQoN2+t8yI2Ogpf3tfPVdJVXXrAk/aNagHw3XPy8ZH+hV5htYFpxloCbtcZtoy5hEyohRAzhRBpQoi0lBRjs+G04PM4chjV5RIMM7k2eLfmjun1Ve0I8+IfuwAAeqfqhx+6N09Gh0scgu5ZG33rsyPcwinNnXFpXygWVw6gUsiF2qSikn4xkmHDhA65Jrxw5COiePdW77rT/gg0vPXxXX2QmXvecEz8q/v6IVGj4FNaaj3NTj0/PTTQLY97+o3dMXrGalcGyed/vRy/Z59DnZqxaJbs8Nj7ONue/fTQQKzcq//0qKXJfxnYSnOd7x8cgGsDaMFVuQ+y5MK5skNDLDFQg7lFvQRX/QwmeOQSagifFbAY+2D071ynZqxbNog/emv0jlz7hHdqn4JnL8x4pycd68wg6d+mAfq3cZ+sc2XHhq51ffXSVFINK3PH4fLYG9aKx0lVP79WDRJ1t+OL6Chya0kVCO/+uSfunx1cZxKjDwbseYcGI+l5cwCsBdCeiI4S0d1mGsShDyZUXNa0Dh67ur3P2LUniuhpefAVPuLlWijbEBqJp4FK6/Qbu7m97+YsXvbRnb0D3FLwNqgZ39t/bRMAuNlADRTGP349aiHEzeEwxLGvcO2JsYpw/ol/fCjw/pZlzmRnrd6USgOHhAArCao96oa14nFn/1Tc1Lu5o1rkc0tcN4d5Dw7w6kD+2NXtcWNac6TUikfbhkm45m3H9/P+VvnbZt7Wy61Bq1GqMgFHqXLpj2CfFBh35JqZCPaobY/kN+Nm9Rze931DWnt9d9+QNpg8qgNuDrDqnzqfmogw9ZrO6Ni4NhLjY9x6U3ZrnowtT1dO6Nn67Ag8OLStK17uWbdcYUTnS7D7Bf9THYZ3dB/4vaJ9Q5+Dq74w2r5O3QZ1QNv6aJps/OmGqUQqoQaMxy6ZyKS0wpGrHBcj59+5do1YZE0b4+p6r6ZGbDTuG9JG09v2hZLpYeQeVTcxDm+O7447+6eiTk3vPO6Xr78Mb47vrmmbwtf39cPbt/TA7hdG4kZnPffkhFivJ9ak+Bh8dZ/vAlgKM252nx5fK4gc8ygizVIAkYS6d2g4kUqoecKL/VEmlcQFKHaRjCJWRkMN47o3xdRrOmt+d1PvFpo3EQC4tFES0lrWRVpqPYzt2gQ1YqPxyg3d8NqfumHegwNwiar5hl63mf0vjsKnf/EuXNVcQ6D0miWMVDU3Vqcj2mFgceZtaZbsV6qrRQA848XmuIRa0s7vZqBolc/kjBD4KEseGYKvNUrEXt+rGVrWT8TTYzvhzfHdcfCl0Zh5u7bgxERHYfCl3vMgKgQw+jKHAL/7Z0ddFK1mCQdfGo1b+7Z0vVdfzkZTLK/t3gTpTw3HBzo2At6Dq+EgNtp/bXwjhcGCQar0PIB12u6UljuEOtbGHvXyvw9xawYc5YpRazTGCJdRcIRH9Lxx/wi8fXNPVIwXXqGfXi3rYtMhRwssIsLAdg3QvXkyMo7ku3nUepOWasZGo6i03PV+ZJfGaJAU7xov8KRuQizaNaxl2PIb05rhy/SjhpfXY+Gkwa6B37YNk7D/pHdn84Q4cxpnyHW1cOTD9lQHj7p1SpLboJkiVhUaLvW067uiRb0EJNWw3md66Mq2rtfqErMjOjXCZU2TERVFXiK9+vGhmqEShShy5LHXS4zDM2M7uX03vGNDbJgyDBnPXuWaRARUPoHojVf9/OgQxBoc4+jXuj7+ea17iEYdmgmEtg2T/HrUZpWylepqcWR9sE9tZ3o763Jc3qr6FCdyCbXGNTz6ssZY9fjQoKsRhoLZ91yOxLhoPKyqJhgVRfjkL33w498GYubtabo31ub1Elxpi2qUn0pEaFynJjY/fZXXtPsP7uiNhrVqID4mGiM6V8bMleN1aaMkzX3WT4p31W7R4roelU8Ncyb0RWw0Yaxq+v9//uy/pO039/fTnMka5QpjaQtyRYXmx1XG+tu4ByzT9mZQuxTseO7qkHU1jwTIz8VtNQPaNsDO573T+4ZoxKr1GNe9iWbHe6P3n7sHtsK6zFP4eddJV50VIsK+F0eh3ZSFXssTEbKmjUHq5PkAgHsGtsJDV7ZD9tkitE5JRHFZhas1GxHh7Vt6YvqNDhVVl8NNTohFlyZ1sGZ/ntv2e7V0n+HauUltAJVNn8de1hgzlu8H4Gg3N/DlFQDMS4iQy6OW9ERmQkt1EmlAlZ5n49P7zfE9NDvmGH1CJiJMv6k7nhrTEQNV/TWNjmX84+r2qJMQi46NayM+Jhrv/Lkn3hzvnlIYFxPlejJQys1OvLIdujV3TN75/J7Ldbc/f+IgAEByQhy2Tx3h9vTRrG6Ca936SdVgMJEnvDB2RD0zsboRSESndo1Y3DPIe6KRL7o1T8blreppNlb2xdQ/dMbtH27AmK6NUS8xDmkt66F/2wbY88+Rbp2CVj02FIWl7h3LtXLI+7dtgCGXpiC/sCQgO4wil1BzF3LGhrhqhFQnpfbxm1++/jIsNVB5z5MV/7jCqxLfvAcHBGVeu0a1sPaJyn6WQ53leuNj3AW/RX3/5WwViMzLh5BKqBnGjighPRvM9zCMIlhav/mm3i1wk8GiTgDwxKgO6Ni4Nlo1SJSudsioLpUZJATzwltSCbWAqF5eB1MtUAag6iXG+VnSfoTier53SJsQWBJ6PMMkUUSuSomhRi6h5tAHY0Puu6INmiTXxDXdmlhtStgoLnVkWNj5evYMkxCZl54nVdYHAHv/ZZlqSWx0FK7v1axaPS3uOVEAADh+tshiS8IJmRajlkqoKyoEN/9kGBsw5jLHBBNPr9POOLqiWZhHTUQjiWgPEe0nosmmWAKgoLis2uXYMowdmX5TN7x6Q1dc0T74RteRRpSJ7SuNtOKKBvAOgFEAOgG4mYg6+V4rOIZ1aGi4cwTDMPISHxONP6U1r1bhnriYaJSUmxOkNuK+9gGwXwiRCQBENBfAOAC/h9qYNzxmEjEMw0QKNWOjUFRS7n/BIDAi1E0BHFG9PwrAa64lEU0AMAEAWrTghpYMw1Qvujeva9oYm5EYtdaevSIxQoiZQog0IURaSkr1iUsxDMMAwC2Xt8C067uasm0jQn0UgLqbZzMA2aZYwzAMw3hhRKg3AmhHRK2IKA7AeAA/mGsWwzAMo+A3Ri2EKCOivwFYDCAawIdCiJ2mW8YwDMMAMDiFXAixAMACk21hGIZhNJBqZiLDMAzjDQs1wzCM5LBQMwzDSA4LNcMwjOSQGdWeiCgXwKEgV28AIM/vUtYTKXYCbKsZRIqdANtqBmbY2VIIoTlb0BShrgpElC6ESLPaDn9Eip0A22oGkWInwLaaQbjt5NAHwzCM5LBQMwzDSI6MQj3TagMMEil2AmyrGUSKnQDbagZhtVO6GDXDMAzjjoweNcMwDKOChZphGEZypBHqcDXQDQQiyiKi7USUQUTpzs/qEdFSItrn/L+uavknnPbvIaKrTbTrQyI6SUQ7VJ8FbBcR9XL+vv1ENINMaHCnY+tUIjrmPK4ZRDTaaluJqDkRrSCiXUS0k4gmOT+X7rj6sFXG41qDiDYQ0Vanrc85P5fquPqwU45jKoSw/B8c5VMPAGgNIA7AVgCdJLArC0ADj89eATDZ+XoygJedrzs57Y4H0Mr5e6JNsmswgJ4AdlTFLgAbAPSDo4vPQgCjwmTrVAD/0FjWMlsBNAbQ0/m6FoC9TnukO64+bJXxuBKAJOfrWADrAfSV7bj6sFOKYyqLR+1qoCuEKAGgNNCVkXEAPnG+/gTAtarP5wohioUQBwHsh+N3hRwhxCoAp6tiFxE1BlBbCLFWOM6uT1XrmG2rHpbZKoQ4LoTY7HxdAGAXHP1CpTuuPmzVw0pbhRDivPNtrPOfgGTH1YedeoTVTlmEWquBrq8TL1wIAEuIaBM5mvcCQCMhxHHAccEAaOj83OrfEKhdTZ2vPT8PF38jom3O0Ijy2CuFrUSUCqAHHF6V1MfVw1ZAwuNKRNFElAHgJIClQggpj6uOnYAEx1QWoTbUQNcCBgghegIYBeBBIhrsY1lZf4OeXVba+y6ANgC6AzgO4DXn55bbSkRJAL4B8LAQ4pyvRXVsstJWKY+rEKJcCNEdjn6rfYioi4/FLbNVx04pjqksQi1lA10hRLbz/5MAvoMjlHHC+XgD5/8nnYtb/RsCteuo87Xn56YjhDjhvCgqAPwXlSEiS20lolg4hG+2EOJb58dSHlctW2U9rgpCiHwAvwAYCUmPq6edshxTWYRauga6RJRIRLWU1wBGANjhtOsO52J3AJjnfP0DgPFEFE9ErQC0g2NQIVwEZJfzcbOAiPo6R6VvV61jKsoF6uSPcBxXS211bncWgF1CiOmqr6Q7rnq2SnpcU4go2fm6JoDhAHZDsuOqZ6c0x7Sqo5Gh+gdgNByj1wcATJHAntZwjOpuBbBTsQlAfQDLAOxz/l9Ptc4Up/17YEIGhWo/c+B4DCuF4w5+dzB2AUhznngHALwN50zVMNj6GYDtALY5T/jGVtsKYCAcj6jbAGQ4/42W8bj6sFXG49oVwBanTTsAPBPsdWSmrT7slOKY8hRyhmEYyZEl9MEwDMPowELNMAwjOSzUDMMwksNCzTAMIzks1AzDMJLDQs0wDCM5LNQMwzCS8/87ET/K0fpgcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "173.com﻿另一只眼睛，每一士所能明\n",
      "　　道：“范大夫，你打断了它。”后来回\n",
      "　　他的乡下姑娘，我要杀我来找我，出来没去找他的一声来。”阿青将羊快越阿青将过了个高坐了。”\n",
      "　　十二三次，要杀要杀了风。青衣\n",
      "　　王明亮如秋水里。他\n",
      "　　纤纤，连鼻子也不能随少便欺侮人。\n",
      "　　这少女子扑倒说过力来，只得对手招数已成一\n",
      "　　。\n",
      "　　十见到欧治子到她但一起了手，浅绿色布衫，直在会的脸上了神情坦率，而退。\n"
     ]
    }
   ],
   "source": [
    "from util import *\n",
    "from scipy.sparse import csr_matrix\n",
    "from Model import nn_LSTM\n",
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "import os\n",
    "import argparse\n",
    "\n",
    "\n",
    "print(\"数据集大小：\", X_train.shape, y_train.shape)\n",
    "\n",
    "is_test = False\n",
    "hidden_size = 256\n",
    "seq_length = 25\n",
    "EPOCHS = 5\n",
    "\n",
    "\n",
    "rnn = nn_LSTM(vocab_size, hidden_size, vocab_size)\n",
    "\n",
    "# 看是否有训练差不多的模型\n",
    "if os.path.exists(\"source/save_model.pth\"):\n",
    "    print(\"finetuning!\")\n",
    "    checkpoint = torch.load(\"source/save_model.pth\")\n",
    "    rnn.load_state_dict(checkpoint)\n",
    "    # rnn.load_state_dict(torch.load('source/save_model.pth'))\n",
    "\n",
    "if is_test:  # 直接测试\n",
    "    for batch in get_batch(X_train, y_train, seq_length):\n",
    "        X_batch, y_batch = batch\n",
    "    rnn.load_state_dict(torch.load('source/save_model.pth'))\n",
    "    print(sample_chars(rnn, X_batch[0], rnn.initHidden_test(), chars, 200))\n",
    "\n",
    "else:# 开始训练过程\n",
    "    rnn = rnn.cuda()  # 使用gpu\n",
    "    loss_fn = nn.CrossEntropyLoss().cuda()\n",
    "    optimizer = torch.optim.Adam([{'params': rnn.parameters(), 'initial_lr': 0.05}], lr=0.05)\n",
    "\n",
    "    scheduler = LambdaLR(optimizer, lr_lambda=lambda epoch: 1 / (epoch + 1))    # 训练模型\n",
    "    all_losses = []\n",
    "    print_every = 500\n",
    "    for epoch in range(EPOCHS):\n",
    "        print(\"当前学习率：\", optimizer.param_groups[0]['lr'])\n",
    "        for batch in get_batch(X_train, y_train, seq_length):\n",
    "            X_batch, y_batch = batch\n",
    "            # print(X_batch, y_batch)\n",
    "            X_batch, y_batch = X_batch.cuda(), y_batch.cuda()\n",
    "            _, batch_loss = train(X_batch, y_batch, rnn, optimizer, loss_fn)\n",
    "            batch_loss = batch_loss.cpu()\n",
    "            all_losses.append(batch_loss.item())\n",
    "            if len(all_losses) % print_every == 1:\n",
    "                print(\"epoch:\", epoch+1, f'----\\nRunning Avg Loss:{np.mean(all_losses[-print_every:])} at iter: {len(all_losses)}\\n----')\n",
    "                print(sample_chars(rnn, X_batch[0], rnn.initHidden(), chars, 200))\n",
    "        scheduler.step()\n",
    "        # 每过一个epoch保存一次模型\n",
    "        torch.save(rnn.state_dict(), 'source/save_model.pth')\n",
    "    print(\"训练完成\")\n",
    "    plot_loss(all_losses)\n",
    "    print(sample_chars(rnn, X_batch[0], rnn.initHidden(), chars, 200))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
